{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generate train/dev/test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_path_name = 'four_category'\n",
    "# target_path_name = 'four_category_G'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 01 generate ids\n",
    " ( delimeter = space )\n",
    "- line[0] = train\n",
    "- line[1] = dev\n",
    "- line[2] = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5531"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "lines = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    lines = f.readlines()\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(337)\n",
    "\n",
    "cfold01_test = []\n",
    "cfold02_test = []\n",
    "cfold03_test = []\n",
    "cfold04_test = []\n",
    "cfold05_test = []\n",
    "\n",
    "for x in xrange( len(lines) ) :\n",
    "    \n",
    "    compare = random.random()\n",
    "    \n",
    "    if compare > 0.8 : \n",
    "        cfold01_test.append( str(x) )\n",
    "    elif compare > 0.6 : \n",
    "        cfold02_test.append( str(x) )\n",
    "    elif compare > 0.4 : \n",
    "        cfold03_test.append( str(x) )\n",
    "    elif compare > 0.2 : \n",
    "        cfold04_test.append( str(x) )        \n",
    "    else:\n",
    "        cfold05_test.append( str(x) )        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_data( list_all, list_test, path ) :\n",
    "\n",
    "    random.seed(3372)\n",
    "    \n",
    "    train = []\n",
    "    dev = []\n",
    "    test = []\n",
    "\n",
    "    for index in xrange( len(list_all) ) :\n",
    "\n",
    "        compare = random.random()\n",
    "\n",
    "        if str(index) not in list_test:\n",
    "            train.append( str(index) )\n",
    "        else:\n",
    "            if compare > 0.70 : \n",
    "                dev.append( str(index) )\n",
    "            else:\n",
    "                test.append( str(index) )\n",
    "\n",
    "    with open(path, 'w') as f :\n",
    "        f.write( ' '.join(train) + '\\n')\n",
    "        f.write( ' '.join(dev) + '\\n')\n",
    "        f.write( ' '.join(test) + '\\n')\n",
    "\n",
    "    print len(train)\n",
    "    print len(dev)\n",
    "    print len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from file_util import *\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path_name+'/audio_woZ_set01')\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path_name+'/audio_woZ_set02')\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path_name+'/audio_woZ_set03')\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path_name+'/audio_woZ_set04')\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path_name+'/audio_woZ_set05')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4445\n",
      "313\n",
      "773\n",
      "4374\n",
      "386\n",
      "771\n",
      "4399\n",
      "353\n",
      "779\n",
      "4432\n",
      "343\n",
      "756\n",
      "4474\n",
      "307\n",
      "750\n"
     ]
    }
   ],
   "source": [
    "path = '../data/processed/IEMOCAP/' + target_path_name + '/audio_woZ_set01/audio_woZ_set01.txt'\n",
    "gen_data( lines, cfold01_test, path )\n",
    "\n",
    "path = '../data/processed/IEMOCAP/' + target_path_name + '/audio_woZ_set02/audio_woZ_set02.txt'\n",
    "gen_data( lines, cfold02_test, path )\n",
    "\n",
    "path = '../data/processed/IEMOCAP/' + target_path_name + '/audio_woZ_set03/audio_woZ_set03.txt'\n",
    "gen_data( lines, cfold03_test, path )\n",
    "\n",
    "path = '../data/processed/IEMOCAP/' + target_path_name + '/audio_woZ_set04/audio_woZ_set04.txt'\n",
    "gen_data( lines, cfold04_test, path )\n",
    "\n",
    "path = '../data/processed/IEMOCAP/' + target_path_name + '/audio_woZ_set05/audio_woZ_set05.txt'\n",
    "gen_data( lines, cfold05_test, path )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 02 generate data according to the ids\n",
    "- train(dev/test)_audio_mfcc.npy\n",
    "- train(dev/test)_audio_prosody.npy\n",
    "- train(dev/test)_audio_seqN.npy\n",
    "- train(dev/test)_label.npy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def extract_data_with_ids( npy_data, ids ) :\n",
    "    npy_data_select = npy_data[ids][:][:]\n",
    "    print np.shape(npy_data_select)\n",
    "    return npy_data_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4445\n",
      "313\n",
      "773\n",
      "(4445, 750, 39)\n",
      "(313, 750, 39)\n",
      "(773, 750, 39)\n",
      "(4445, 35)\n",
      "(313, 35)\n",
      "(773, 35)\n",
      "(4445,)\n",
      "(313,)\n",
      "(773,)\n",
      "(4445,)\n",
      "(313,)\n",
      "(773,)\n",
      "(4445, 128)\n",
      "(313, 128)\n",
      "(773, 128)\n"
     ]
    }
   ],
   "source": [
    "target_name = 'audio_woZ_set01'\n",
    "\n",
    "# word embedding link\n",
    "cmd = 'ln -s ../../W_embedding.npy ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "# dic link\n",
    "cmd = 'ln -s ../../dic.pkl ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "\n",
    "target_sequence = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/' + target_name + '.txt'\n",
    "target_path = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/'\n",
    "\n",
    "train_ids = [] \n",
    "dev_ids = [] \n",
    "test_ids = [] \n",
    "with open( target_sequence ) as f:\n",
    "    lines = f.readlines()\n",
    "    train_ids = [ int(x) for x in lines[0].strip().split(' ')]\n",
    "    dev_ids =  [ int(x) for x in lines[1].strip().split(' ')]\n",
    "    test_ids = [ int(x) for x in lines[2].strip().split(' ')]\n",
    "\n",
    "print len(train_ids)\n",
    "print len(dev_ids)\n",
    "print len(test_ids)\n",
    "\n",
    "\n",
    "# MFCC\n",
    "train_audio_mfcc = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), train_ids  )\n",
    "dev_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), dev_ids  )\n",
    "test_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_mfcc.npy', train_audio_mfcc)\n",
    "np.save( target_path + 'dev_audio_mfcc.npy', dev_audio_mfcc)\n",
    "np.save( target_path + 'test_audio_mfcc.npy', test_audio_mfcc)\n",
    "\n",
    "\n",
    "# prosody\n",
    "train_audio_prosody = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), train_ids  )\n",
    "dev_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), dev_ids  )\n",
    "test_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_prosody.npy', train_audio_prosody)\n",
    "np.save( target_path + 'dev_audio_prosody.npy', dev_audio_prosody)\n",
    "np.save( target_path + 'test_audio_prosody.npy', test_audio_prosody)\n",
    "\n",
    "\n",
    "# emobase2010\n",
    "# train_audio_emobase2010 = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), train_ids  )\n",
    "# dev_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), dev_ids  )\n",
    "# test_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), test_ids  )\n",
    "\n",
    "# np.save( target_path + 'train_audio_emobase2010.npy', train_audio_emobase2010)\n",
    "# np.save( target_path + 'dev_audio_emobase2010.npy', dev_audio_emobase2010)\n",
    "# np.save( target_path + 'test_audio_emobase2010.npy', test_audio_emobase2010)\n",
    "\n",
    "\n",
    "# sequenceN\n",
    "seqN_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA_sequenceN.txt') as f :\n",
    "    seqN = [ int(x.strip()) for x in f.readlines() ]\n",
    "    seqN_npy = np.asarray(seqN)\n",
    "    \n",
    "train_seqN = extract_data_with_ids( seqN_npy, train_ids  )\n",
    "dev_seqN  = extract_data_with_ids( seqN_npy, dev_ids  )\n",
    "test_seqN = extract_data_with_ids( seqN_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_seqN.npy', train_seqN)\n",
    "np.save( target_path + 'dev_seqN.npy', dev_seqN)\n",
    "np.save( target_path + 'test_seqN.npy', test_seqN)\n",
    "\n",
    "\n",
    "# label\n",
    "label_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    label = [ int(x.strip()) for x in f.readlines() ]\n",
    "    label_npy = np.asarray(label)\n",
    "\n",
    "train_label = extract_data_with_ids( label_npy, train_ids  )\n",
    "dev_label  = extract_data_with_ids( label_npy, dev_ids  )\n",
    "test_label = extract_data_with_ids( label_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_label.npy', train_label)\n",
    "np.save( target_path + 'dev_label.npy', dev_label)\n",
    "np.save( target_path + 'test_label.npy', test_label)\n",
    "\n",
    "# trans\n",
    "train_nlp_trans = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), train_ids  )\n",
    "dev_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), dev_ids  )\n",
    "test_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_nlp_trans.npy', train_nlp_trans)\n",
    "np.save( target_path + 'dev_nlp_trans.npy', dev_nlp_trans)\n",
    "np.save( target_path + 'test_nlp_trans.npy', test_nlp_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4374\n",
      "386\n",
      "771\n",
      "(4374, 750, 39)\n",
      "(386, 750, 39)\n",
      "(771, 750, 39)\n",
      "(4374, 35)\n",
      "(386, 35)\n",
      "(771, 35)\n",
      "(4374,)\n",
      "(386,)\n",
      "(771,)\n",
      "(4374,)\n",
      "(386,)\n",
      "(771,)\n",
      "(4374, 128)\n",
      "(386, 128)\n",
      "(771, 128)\n"
     ]
    }
   ],
   "source": [
    "target_name = 'audio_woZ_set02'\n",
    "\n",
    "# word embedding link\n",
    "cmd = 'ln -s ../../W_embedding.npy ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "# dic link\n",
    "cmd = 'ln -s ../../dic.pkl ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "target_sequence = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/' + target_name + '.txt'\n",
    "target_path = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/'\n",
    "\n",
    "train_ids = [] \n",
    "dev_ids = [] \n",
    "test_ids = [] \n",
    "with open( target_sequence ) as f:\n",
    "    lines = f.readlines()\n",
    "    train_ids = [ int(x) for x in lines[0].strip().split(' ')]\n",
    "    dev_ids =  [ int(x) for x in lines[1].strip().split(' ')]\n",
    "    test_ids = [ int(x) for x in lines[2].strip().split(' ')]\n",
    "\n",
    "print len(train_ids)\n",
    "print len(dev_ids)\n",
    "print len(test_ids)\n",
    "\n",
    "\n",
    "# MFCC\n",
    "train_audio_mfcc = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), train_ids  )\n",
    "dev_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), dev_ids  )\n",
    "test_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_mfcc.npy', train_audio_mfcc)\n",
    "np.save( target_path + 'dev_audio_mfcc.npy', dev_audio_mfcc)\n",
    "np.save( target_path + 'test_audio_mfcc.npy', test_audio_mfcc)\n",
    "\n",
    "\n",
    "# prosody\n",
    "train_audio_prosody = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), train_ids  )\n",
    "dev_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), dev_ids  )\n",
    "test_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_prosody.npy', train_audio_prosody)\n",
    "np.save( target_path + 'dev_audio_prosody.npy', dev_audio_prosody)\n",
    "np.save( target_path + 'test_audio_prosody.npy', test_audio_prosody)\n",
    "\n",
    "\n",
    "# emobase2010\n",
    "# train_audio_emobase2010 = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), train_ids  )\n",
    "# dev_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), dev_ids  )\n",
    "# test_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), test_ids  )\n",
    "\n",
    "# np.save( target_path + 'train_audio_emobase2010.npy', train_audio_emobase2010)\n",
    "# np.save( target_path + 'dev_audio_emobase2010.npy', dev_audio_emobase2010)\n",
    "# np.save( target_path + 'test_audio_emobase2010.npy', test_audio_emobase2010)\n",
    "\n",
    "\n",
    "# sequenceN\n",
    "seqN_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA_sequenceN.txt') as f :\n",
    "    seqN = [ int(x.strip()) for x in f.readlines() ]\n",
    "    seqN_npy = np.asarray(seqN)\n",
    "    \n",
    "train_seqN = extract_data_with_ids( seqN_npy, train_ids  )\n",
    "dev_seqN  = extract_data_with_ids( seqN_npy, dev_ids  )\n",
    "test_seqN = extract_data_with_ids( seqN_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_seqN.npy', train_seqN)\n",
    "np.save( target_path + 'dev_seqN.npy', dev_seqN)\n",
    "np.save( target_path + 'test_seqN.npy', test_seqN)\n",
    "\n",
    "\n",
    "# label\n",
    "label_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    label = [ int(x.strip()) for x in f.readlines() ]\n",
    "    label_npy = np.asarray(label)\n",
    "\n",
    "train_label = extract_data_with_ids( label_npy, train_ids  )\n",
    "dev_label  = extract_data_with_ids( label_npy, dev_ids  )\n",
    "test_label = extract_data_with_ids( label_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_label.npy', train_label)\n",
    "np.save( target_path + 'dev_label.npy', dev_label)\n",
    "np.save( target_path + 'test_label.npy', test_label)\n",
    "\n",
    "# trans\n",
    "train_nlp_trans = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), train_ids  )\n",
    "dev_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), dev_ids  )\n",
    "test_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_nlp_trans.npy', train_nlp_trans)\n",
    "np.save( target_path + 'dev_nlp_trans.npy', dev_nlp_trans)\n",
    "np.save( target_path + 'test_nlp_trans.npy', test_nlp_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4399\n",
      "353\n",
      "779\n",
      "(4399, 750, 39)\n",
      "(353, 750, 39)\n",
      "(779, 750, 39)\n",
      "(4399, 35)\n",
      "(353, 35)\n",
      "(779, 35)\n",
      "(4399,)\n",
      "(353,)\n",
      "(779,)\n",
      "(4399,)\n",
      "(353,)\n",
      "(779,)\n",
      "(4399, 128)\n",
      "(353, 128)\n",
      "(779, 128)\n"
     ]
    }
   ],
   "source": [
    "target_name = 'audio_woZ_set03'\n",
    "\n",
    "# word embedding link\n",
    "cmd = 'ln -s ../../W_embedding.npy ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "# dic link\n",
    "cmd = 'ln -s ../../dic.pkl ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "target_sequence = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/' + target_name + '.txt'\n",
    "target_path = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/'\n",
    "\n",
    "train_ids = [] \n",
    "dev_ids = [] \n",
    "test_ids = [] \n",
    "with open( target_sequence ) as f:\n",
    "    lines = f.readlines()\n",
    "    train_ids = [ int(x) for x in lines[0].strip().split(' ')]\n",
    "    dev_ids =  [ int(x) for x in lines[1].strip().split(' ')]\n",
    "    test_ids = [ int(x) for x in lines[2].strip().split(' ')]\n",
    "\n",
    "print len(train_ids)\n",
    "print len(dev_ids)\n",
    "print len(test_ids)\n",
    "\n",
    "\n",
    "# MFCC\n",
    "train_audio_mfcc = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), train_ids  )\n",
    "dev_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), dev_ids  )\n",
    "test_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_mfcc.npy', train_audio_mfcc)\n",
    "np.save( target_path + 'dev_audio_mfcc.npy', dev_audio_mfcc)\n",
    "np.save( target_path + 'test_audio_mfcc.npy', test_audio_mfcc)\n",
    "\n",
    "\n",
    "# prosody\n",
    "train_audio_prosody = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), train_ids  )\n",
    "dev_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), dev_ids  )\n",
    "test_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_prosody.npy', train_audio_prosody)\n",
    "np.save( target_path + 'dev_audio_prosody.npy', dev_audio_prosody)\n",
    "np.save( target_path + 'test_audio_prosody.npy', test_audio_prosody)\n",
    "\n",
    "\n",
    "# emobase2010\n",
    "# train_audio_emobase2010 = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), train_ids  )\n",
    "# dev_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), dev_ids  )\n",
    "# test_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), test_ids  )\n",
    "\n",
    "# np.save( target_path + 'train_audio_emobase2010.npy', train_audio_emobase2010)\n",
    "# np.save( target_path + 'dev_audio_emobase2010.npy', dev_audio_emobase2010)\n",
    "# np.save( target_path + 'test_audio_emobase2010.npy', test_audio_emobase2010)\n",
    "\n",
    "\n",
    "# sequenceN\n",
    "seqN_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA_sequenceN.txt') as f :\n",
    "    seqN = [ int(x.strip()) for x in f.readlines() ]\n",
    "    seqN_npy = np.asarray(seqN)\n",
    "    \n",
    "train_seqN = extract_data_with_ids( seqN_npy, train_ids  )\n",
    "dev_seqN  = extract_data_with_ids( seqN_npy, dev_ids  )\n",
    "test_seqN = extract_data_with_ids( seqN_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_seqN.npy', train_seqN)\n",
    "np.save( target_path + 'dev_seqN.npy', dev_seqN)\n",
    "np.save( target_path + 'test_seqN.npy', test_seqN)\n",
    "\n",
    "\n",
    "# label\n",
    "label_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    label = [ int(x.strip()) for x in f.readlines() ]\n",
    "    label_npy = np.asarray(label)\n",
    "\n",
    "train_label = extract_data_with_ids( label_npy, train_ids  )\n",
    "dev_label  = extract_data_with_ids( label_npy, dev_ids  )\n",
    "test_label = extract_data_with_ids( label_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_label.npy', train_label)\n",
    "np.save( target_path + 'dev_label.npy', dev_label)\n",
    "np.save( target_path + 'test_label.npy', test_label)\n",
    "\n",
    "# trans\n",
    "train_nlp_trans = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), train_ids  )\n",
    "dev_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), dev_ids  )\n",
    "test_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_nlp_trans.npy', train_nlp_trans)\n",
    "np.save( target_path + 'dev_nlp_trans.npy', dev_nlp_trans)\n",
    "np.save( target_path + 'test_nlp_trans.npy', test_nlp_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4432\n",
      "343\n",
      "756\n",
      "(4432, 750, 39)\n",
      "(343, 750, 39)\n",
      "(756, 750, 39)\n",
      "(4432, 35)\n",
      "(343, 35)\n",
      "(756, 35)\n",
      "(4432,)\n",
      "(343,)\n",
      "(756,)\n",
      "(4432,)\n",
      "(343,)\n",
      "(756,)\n",
      "(4432, 128)\n",
      "(343, 128)\n",
      "(756, 128)\n"
     ]
    }
   ],
   "source": [
    "target_name = 'audio_woZ_set04'\n",
    "\n",
    "# word embedding link\n",
    "cmd = 'ln -s ../../W_embedding.npy ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "# dic link\n",
    "cmd = 'ln -s ../../dic.pkl ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "target_sequence = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/' + target_name + '.txt'\n",
    "target_path = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/'\n",
    "\n",
    "train_ids = [] \n",
    "dev_ids = [] \n",
    "test_ids = [] \n",
    "with open( target_sequence ) as f:\n",
    "    lines = f.readlines()\n",
    "    train_ids = [ int(x) for x in lines[0].strip().split(' ')]\n",
    "    dev_ids =  [ int(x) for x in lines[1].strip().split(' ')]\n",
    "    test_ids = [ int(x) for x in lines[2].strip().split(' ')]\n",
    "\n",
    "print len(train_ids)\n",
    "print len(dev_ids)\n",
    "print len(test_ids)\n",
    "\n",
    "\n",
    "# MFCC\n",
    "train_audio_mfcc = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), train_ids  )\n",
    "dev_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), dev_ids  )\n",
    "test_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_mfcc.npy', train_audio_mfcc)\n",
    "np.save( target_path + 'dev_audio_mfcc.npy', dev_audio_mfcc)\n",
    "np.save( target_path + 'test_audio_mfcc.npy', test_audio_mfcc)\n",
    "\n",
    "\n",
    "# prosody\n",
    "train_audio_prosody = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), train_ids  )\n",
    "dev_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), dev_ids  )\n",
    "test_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_prosody.npy', train_audio_prosody)\n",
    "np.save( target_path + 'dev_audio_prosody.npy', dev_audio_prosody)\n",
    "np.save( target_path + 'test_audio_prosody.npy', test_audio_prosody)\n",
    "\n",
    "\n",
    "# emobase2010\n",
    "# train_audio_emobase2010 = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), train_ids  )\n",
    "# dev_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), dev_ids  )\n",
    "# test_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), test_ids  )\n",
    "\n",
    "# np.save( target_path + 'train_audio_emobase2010.npy', train_audio_emobase2010)\n",
    "# np.save( target_path + 'dev_audio_emobase2010.npy', dev_audio_emobase2010)\n",
    "# np.save( target_path + 'test_audio_emobase2010.npy', test_audio_emobase2010)\n",
    "\n",
    "\n",
    "# sequenceN\n",
    "seqN_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA_sequenceN.txt') as f :\n",
    "    seqN = [ int(x.strip()) for x in f.readlines() ]\n",
    "    seqN_npy = np.asarray(seqN)\n",
    "    \n",
    "train_seqN = extract_data_with_ids( seqN_npy, train_ids  )\n",
    "dev_seqN  = extract_data_with_ids( seqN_npy, dev_ids  )\n",
    "test_seqN = extract_data_with_ids( seqN_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_seqN.npy', train_seqN)\n",
    "np.save( target_path + 'dev_seqN.npy', dev_seqN)\n",
    "np.save( target_path + 'test_seqN.npy', test_seqN)\n",
    "\n",
    "\n",
    "# label\n",
    "label_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    label = [ int(x.strip()) for x in f.readlines() ]\n",
    "    label_npy = np.asarray(label)\n",
    "\n",
    "train_label = extract_data_with_ids( label_npy, train_ids  )\n",
    "dev_label  = extract_data_with_ids( label_npy, dev_ids  )\n",
    "test_label = extract_data_with_ids( label_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_label.npy', train_label)\n",
    "np.save( target_path + 'dev_label.npy', dev_label)\n",
    "np.save( target_path + 'test_label.npy', test_label)\n",
    "\n",
    "# trans\n",
    "train_nlp_trans = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), train_ids  )\n",
    "dev_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), dev_ids  )\n",
    "test_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_nlp_trans.npy', train_nlp_trans)\n",
    "np.save( target_path + 'dev_nlp_trans.npy', dev_nlp_trans)\n",
    "np.save( target_path + 'test_nlp_trans.npy', test_nlp_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4474\n",
      "307\n",
      "750\n",
      "(4474, 750, 39)\n",
      "(307, 750, 39)\n",
      "(750, 750, 39)\n",
      "(4474, 35)\n",
      "(307, 35)\n",
      "(750, 35)\n",
      "(4474,)\n",
      "(307,)\n",
      "(750,)\n",
      "(4474,)\n",
      "(307,)\n",
      "(750,)\n",
      "(4474, 128)\n",
      "(307, 128)\n",
      "(750, 128)\n"
     ]
    }
   ],
   "source": [
    "target_name = 'audio_woZ_set05'\n",
    "\n",
    "# word embedding link\n",
    "cmd = 'ln -s ../../W_embedding.npy ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "# dic link\n",
    "cmd = 'ln -s ../../dic.pkl ../data/processed/IEMOCAP/four_category/'+target_name+'/'\n",
    "os.system(cmd)\n",
    "\n",
    "target_sequence = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/' + target_name + '.txt'\n",
    "target_path = '../data/processed/IEMOCAP/' + target_path_name + '/' + target_name + '/'\n",
    "\n",
    "train_ids = [] \n",
    "dev_ids = [] \n",
    "test_ids = [] \n",
    "with open( target_sequence ) as f:\n",
    "    lines = f.readlines()\n",
    "    train_ids = [ int(x) for x in lines[0].strip().split(' ')]\n",
    "    dev_ids =  [ int(x) for x in lines[1].strip().split(' ')]\n",
    "    test_ids = [ int(x) for x in lines[2].strip().split(' ')]\n",
    "\n",
    "print len(train_ids)\n",
    "print len(dev_ids)\n",
    "print len(test_ids)\n",
    "\n",
    "\n",
    "# MFCC\n",
    "train_audio_mfcc = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), train_ids  )\n",
    "dev_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), dev_ids  )\n",
    "test_audio_mfcc  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_mfcc.npy', train_audio_mfcc)\n",
    "np.save( target_path + 'dev_audio_mfcc.npy', dev_audio_mfcc)\n",
    "np.save( target_path + 'test_audio_mfcc.npy', test_audio_mfcc)\n",
    "\n",
    "\n",
    "# prosody\n",
    "train_audio_prosody = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), train_ids  )\n",
    "dev_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), dev_ids  )\n",
    "test_audio_prosody  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_prodosy.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_audio_prosody.npy', train_audio_prosody)\n",
    "np.save( target_path + 'dev_audio_prosody.npy', dev_audio_prosody)\n",
    "np.save( target_path + 'test_audio_prosody.npy', test_audio_prosody)\n",
    "\n",
    "\n",
    "# emobase2010\n",
    "# train_audio_emobase2010 = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), train_ids  )\n",
    "# dev_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), dev_ids  )\n",
    "# test_audio_emobase2010  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_emobase2010.npy' ), test_ids  )\n",
    "\n",
    "# np.save( target_path + 'train_audio_emobase2010.npy', train_audio_emobase2010)\n",
    "# np.save( target_path + 'dev_audio_emobase2010.npy', dev_audio_emobase2010)\n",
    "# np.save( target_path + 'test_audio_emobase2010.npy', test_audio_emobase2010)\n",
    "\n",
    "\n",
    "# sequenceN\n",
    "seqN_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_MFCC12EDA_sequenceN.txt') as f :\n",
    "    seqN = [ int(x.strip()) for x in f.readlines() ]\n",
    "    seqN_npy = np.asarray(seqN)\n",
    "    \n",
    "train_seqN = extract_data_with_ids( seqN_npy, train_ids  )\n",
    "dev_seqN  = extract_data_with_ids( seqN_npy, dev_ids  )\n",
    "test_seqN = extract_data_with_ids( seqN_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_seqN.npy', train_seqN)\n",
    "np.save( target_path + 'dev_seqN.npy', dev_seqN)\n",
    "np.save( target_path + 'test_seqN.npy', test_seqN)\n",
    "\n",
    "\n",
    "# label\n",
    "label_npy = []\n",
    "with open('../data/processed/IEMOCAP/' + target_path_name + '/FC_label.txt') as f :\n",
    "    label = [ int(x.strip()) for x in f.readlines() ]\n",
    "    label_npy = np.asarray(label)\n",
    "\n",
    "train_label = extract_data_with_ids( label_npy, train_ids  )\n",
    "dev_label  = extract_data_with_ids( label_npy, dev_ids  )\n",
    "test_label = extract_data_with_ids( label_npy, test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_label.npy', train_label)\n",
    "np.save( target_path + 'dev_label.npy', dev_label)\n",
    "np.save( target_path + 'test_label.npy', test_label)\n",
    "\n",
    "# trans\n",
    "train_nlp_trans = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), train_ids  )\n",
    "dev_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), dev_ids  )\n",
    "test_nlp_trans  = extract_data_with_ids( np.load( '../data/processed/IEMOCAP/' + target_path_name + '/FC_trans.npy' ), test_ids  )\n",
    "\n",
    "np.save( target_path + 'train_nlp_trans.npy', train_nlp_trans)\n",
    "np.save( target_path + 'dev_nlp_trans.npy', dev_nlp_trans)\n",
    "np.save( target_path + 'test_nlp_trans.npy', test_nlp_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf14_p27]",
   "language": "python",
   "name": "conda-env-tf14_p27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
